package us.codecraft.webmagic.sample;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.monitor.SpiderMonitor;
import us.codecraft.webmagic.pipeline.JsonFilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import javax.management.JMException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

/**
 * 微信搜狗公众号搜索爬取
 *
 * @Author: xu.pengfei
 * @Email: faberxu@gmail.com
 * @Date: 2018-08-06 11:18
 */
public class WeixinSougouPageProcessor implements PageProcessor {

    /**
     * 微信公众号最近10条信息列表页面
     * https://mp.weixin.qq.com/profile?src=3&timestamp=1533524977&ver=1&signature=pv3InP2aNnYw1GSwru42*vOOEZveLxHL3wJe6Zt0hJsI*aTjPAlmSe1QBX*IFCvRbX9NGNazRZ-D21kRP18CiQ==
     */
    public static final String URL_WX_PROFILE = "http://mp\\.weixin\\.qq\\.com/profile.*";
    /**
     * 微信公众号文章页面
     */
    public static final String URL_WX_ARTICLE = "https://mp\\.weixin\\.qq\\.com/s.*";

    private Site site = Site
            .me()
            .setDomain("weixin.sogou.com")
            .setSleepTime(3000)
            .setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");

    @Override
    public void process(Page page) {
//        System.out.println(page.getHtml());
        List<String> list = page.getHtml().links().regex("http://mp\\.weixin\\.qq\\.com/profile.*").all();
//        System.out.println(list);
        page.addTargetRequests(list);

        // 匹配微信公众号最近10条信息列表页面
        if (page.getUrl().regex(URL_WX_PROFILE).match()) {
            // 公众号名称
            page.putField("mp_nickname", page.getHtml().xpath("//strong[@class='profile_nickname']//text()").toString());
            // 公众号账户
            page.putField("mp_account", page.getHtml().xpath("//p[@class='profile_account']//text()").toString());
            // 公众号介绍
            List<Selectable> descNodes = page.getHtml().xpath("//ul[@class='profile_desc']//li").nodes();
            JSONArray desc = new JSONArray();
            for (Selectable descNode : descNodes) {
                JSONObject descJo = new JSONObject();
                descJo.put("label", descNode.xpath("//label//text()").toString());
                descJo.put("value", descNode.xpath("//div//text()").toString());
                desc.add(descJo);
            }
            page.putField("desc", desc);

            // 查找最近10条信息array对象
            String rawStr = page.getHtml().regex("var msgList = .*]};").get();
            if (StringUtils.isNotEmpty(rawStr) && rawStr.length() > 14) {
                String jsonStr = rawStr.substring(14, rawStr.length() - 1);
                JSONObject data = JSON.parseObject(jsonStr);
                page.putField("jsonArrayStr", data);

                JSONArray artList = data.getJSONArray("list");
                for (int i = 0; i < artList.size(); i++) {
                    JSONObject arcJo = artList.getJSONObject(i);
                    String artUrl = "https://mp.weixin.qq.com" + arcJo.getJSONObject("app_msg_ext_info").getString("content_url").replaceAll("&amp;", "&");
//                    page.addTargetRequest(artUrl);
                    Request request = new Request(artUrl);
                    request.putExtra("arcJo", arcJo);
                    page.addTargetRequest(request);
                }
            }
        } else if (page.getUrl().regex(URL_WX_ARTICLE).match()) {
//            System.out.println(page.getHtml());
            // 标题
            page.putField("title", page.getHtml().xpath("//h2[@id='activity-name']//text()").toString());
            // 原创
            page.putField("copyright_from", page.getHtml().xpath("//span[@id='copyright_logo']//text()").toString());
            // 作者
            page.putField("author", page.getHtml().xpath("//span[@class='rich_media_meta rich_media_meta_text']//text()").toString());
            // mp公众号
            page.putField("mp", page.getHtml().xpath("//a[@id='js_name']//text()").toString());
            // 文章内容
            page.putField("content", page.getHtml().xpath("//div[@id='js_content']").toString());

            JSONObject arcJo = (JSONObject) page.getRequest().getExtra("arcJo");
            Date createDate = new Date(arcJo.getJSONObject("comm_msg_info").getLong("datetime") * 1000);
            // 发布日期
            page.putField("create_date", DateFormatUtils.format(createDate, "yyyy-MM-dd HH:mm:ss"));
            // 封面图片
            page.putField("cover", arcJo.getString("cover"));
        }
    }

    @Override
    public Site getSite() {
        return site;
    }

    /**
     * 公众号账户数组
     */
    private static final String[] SEARCH_KEYS = new String[]{
            "sgcctop", // 电网头条
            "zgdlb_CEPN", // 中国电力报
            "energyobserver", // 南方能源观察
            "energytalent", // 壹条能
            "chinaden2016", // 分布式能源网
            "Eknower", // 能见Ekonwer
            "PV-2005", // 光伏头条
            "BloombergNEF", // 彭博新能源财经
            "stategridnews", // 国家电网报
            "dwjs1957", // 电网技术
            "bjx-chuneng" // 北极星储能网
    };
    /**
     * 搜狗微信搜索公众号url
     */
    private static final String SEARCH_URL_FORMAT = "http://weixin.sogou.com/weixin?type=1&s_from=input&query=%1$s&ie=utf8&_sug_=n&_sug_type_=";

    public static void main(String[] args) throws JMException, UnsupportedEncodingException {
        List<String> urls = new ArrayList<String>();
        for (String search : SEARCH_KEYS) {
            urls.add(String.format(SEARCH_URL_FORMAT, URLEncoder.encode(search, "utf-8")));
        }
        String[] strings = new String[urls.size()];
        urls.toArray(strings);

        Spider spider = Spider.create(new WeixinSougouPageProcessor())
                .addUrl(strings)
                .addPipeline(new JsonFilePipeline("D:\\webmagic"))
                .thread(5);

        SpiderMonitor.instance().register(spider);

        spider.start();
    }

}
